{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "only (generic function with 1 method)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function only(itr)\n",
    "    state = start(itr)\n",
    "    val,state = next(itr, state)\n",
    "    @assert(done(itr,state))\n",
    "    return val\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "using SymEngine\n",
    "import SymEngine: as_numer_denom\n",
    "using Expr2LaTeX\n",
    "Expr2LaTeX.render(symexpr::SymEngine.Basic)=symexpr |> repr |> parse |> render"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[1m\u001b[32mTest Passed\n",
       "\u001b[39m\u001b[22m"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function ugly_simplify(f)\n",
    "    num,den = as_numer_denom(f)\n",
    "    inv(expand(inv(num)*den))\n",
    "end\n",
    "\n",
    "function basic_simplify(f)\n",
    "    num2,den2 = as_numer_denom(ugly_simplify(f))\n",
    "    num2/den2\n",
    "end\n",
    "\n",
    "(a)≖(b) = ugly_simplify(a)==ugly_simplify(b)\n",
    "\n",
    "### Tests\n",
    "using Base.Test\n",
    "@vars z\n",
    "s1 = (1 + exp(-z))^(-1)\n",
    "s2 = exp(z)/(1+exp(z))\n",
    "\n",
    "@test s1 ≖ s2\n",
    "@test inv(s1)≖inv(s2)\n",
    "\n",
    "n1 = subs(s1, z=>-z)\n",
    "n2 = 1-s1\n",
    "n3 = subs(s2, z=>-z)\n",
    "n4 = 1-s2\n",
    "@test n1 ≖ n2 ≖ n3 ≖ n4\n",
    "@test inv(n1)≖inv(n2) ≖ inv(n3) ≖ inv(n4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sigmoid (generic function with 1 method)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigmoid(z)=(1 + exp(-z))^(-1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "exp(-z)/(1 + exp(-z))^2"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigmoid(z)=(1 + exp(-z))^(-1)\n",
    "@vars z\n",
    "diff(sigmoid(z),z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "vector_var (generic function with 1 method)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function matrix_var(name, input_size, output_size)\n",
    "    [symbols(\"$(name)_$j$i\") for j in 1:output_size, i in 1:input_size]\n",
    "end\n",
    "function vector_var(name, output_size)\n",
    "    [symbols(\"$(name)_$j\") for j in 1:output_size]\n",
    "end\n",
    "\n",
    "const weight_var = matrix_var\n",
    "const bias_var = vector_var\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic Network\n",
    "\n",
    "Single input single output\n",
    "1 hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})$"
      ],
      "text/plain": [
       "L\"$(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "@vars x\n",
    "W = weight_var(\"W\", 1, 3)\n",
    "b = bias_var(\"b\",3)\n",
    "V = weight_var(\"V\", 3, 1)\n",
    "\n",
    "z=(sigmoid.(W*x + b))\n",
    "N=V*z |> only\n",
    "render(N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$(\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * V_{11})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * V_{12})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * V_{13})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}})$"
      ],
      "text/plain": [
       "L\"$(\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * V_{11})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * V_{12})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * V_{13})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dNdx = diff(N,x)\n",
    "render(dNdx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$(\\frac{(W_{11} * V_{11})}{((1 + \\mathrm{exp}(b_{1} + (x * W_{11}))) * (1 + \\mathrm{exp}(-(b_{1} + (x * W_{11})))))} + \\frac{(W_{21} * V_{12})}{((1 + \\mathrm{exp}(b_{2} + (x * W_{21}))) * (1 + \\mathrm{exp}(-(b_{2} + (x * W_{21})))))} + \\frac{(W_{31} * V_{13})}{((1 + \\mathrm{exp}(b_{3} + (x * W_{31}))) * (1 + \\mathrm{exp}(-(b_{3} + (x * W_{31})))))})$"
      ],
      "text/plain": [
       "L\"$(\\frac{(W_{11} * V_{11})}{((1 + \\mathrm{exp}(b_{1} + (x * W_{11}))) * (1 + \\mathrm{exp}(-(b_{1} + (x * W_{11})))))} + \\frac{(W_{21} * V_{12})}{((1 + \\mathrm{exp}(b_{2} + (x * W_{21}))) * (1 + \\mathrm{exp}(-(b_{2} + (x * W_{21})))))} + \\frac{(W_{31} * V_{13})}{((1 + \\mathrm{exp}(b_{3} + (x * W_{31}))) * (1 + \\mathrm{exp}(-(b_{3} + (x * W_{31})))))})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# I am pretty sure this is the right equation for the derivative of a plain nerual network\n",
    "# But symengine is not strong enough to  recognise that it matches the above\n",
    "mdNdx = (W'.*V)*(sigmoid.(-(W*x + b)).*sigmoid.(W*x + b)) |> first\n",
    "render.(mdNdx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1×3 Array{SymEngine.Basic,2}:\n",
       " W_11*V_11  W_21*V_12  W_31*V_13"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W'.*V"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Expodential Output net\n",
    "Single input single output still,\n",
    "1 hidden layer,\n",
    "output expodential,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})$"
      ],
      "text/plain": [
       "L\"$\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "@vars x\n",
    "W = weight_var(\"W\", 1, 3)\n",
    "b = bias_var(\"b\",3)\n",
    "V = weight_var(\"V\", 3, 1)\n",
    "\n",
    "z=sigmoid.(W*x + b)\n",
    "N=exp.(V*z) |> only\n",
    "render(N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$(\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))}) * (\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * V_{11})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * V_{12})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * V_{13})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}}))$"
      ],
      "text/plain": [
       "L\"$(\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))}) * (\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * V_{11})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * V_{12})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * V_{13})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}}))$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dNdx = diff(N,x)\n",
    "render(dNdx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$(\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))}) * (\\frac{(W_{11} * V_{11})}{((1 + \\mathrm{exp}(b_{1} + (x * W_{11}))) * (1 + \\mathrm{exp}(-(b_{1} + (x * W_{11})))))} + \\frac{(W_{21} * V_{12})}{((1 + \\mathrm{exp}(b_{2} + (x * W_{21}))) * (1 + \\mathrm{exp}(-(b_{2} + (x * W_{21})))))} + \\frac{(W_{31} * V_{13})}{((1 + \\mathrm{exp}(b_{3} + (x * W_{31}))) * (1 + \\mathrm{exp}(-(b_{3} + (x * W_{31})))))}))$"
      ],
      "text/plain": [
       "L\"$(\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))}) * (\\frac{(W_{11} * V_{11})}{((1 + \\mathrm{exp}(b_{1} + (x * W_{11}))) * (1 + \\mathrm{exp}(-(b_{1} + (x * W_{11})))))} + \\frac{(W_{21} * V_{12})}{((1 + \\mathrm{exp}(b_{2} + (x * W_{21}))) * (1 + \\mathrm{exp}(-(b_{2} + (x * W_{21})))))} + \\frac{(W_{31} * V_{13})}{((1 + \\mathrm{exp}(b_{3} + (x * W_{31}))) * (1 + \\mathrm{exp}(-(b_{3} + (x * W_{31})))))}))$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# I am pretty sure this is the right equation for the derivative of this network\n",
    "# as we just use the standard `d(exp(f(x))/dx = f'(x)*exp(f(x))` rule\n",
    "$\n",
    "# But symengine is not strong enough to  recognise that it matches the above\n",
    "mdZm1dx = (W'.*V)*(sigmoid.(-(W*x + b)).*sigmoid.(W*x + b)) |> first\n",
    "mdNdx = mdZm1dx*N\n",
    "render.(mdNdx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deeper Expodential Output net\n",
    "Single input single output still,\n",
    "2 hidden layer,\n",
    "output expodential,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(c_{1} + \\frac{U_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(c_{2} + \\frac{U_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(c_{3} + \\frac{U_{31}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{32}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{33}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))})$"
      ],
      "text/plain": [
       "L\"$\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(c_{1} + \\frac{U_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(c_{2} + \\frac{U_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(c_{3} + \\frac{U_{31}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{32}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{33}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "@vars x\n",
    "W = weight_var(\"W\", 1, 3)\n",
    "b = bias_var(\"b\",3)\n",
    "U = weight_var(\"U\", 3, 3)\n",
    "c = bias_var(\"c\",3)\n",
    "V = weight_var(\"V\", 3, 1)\n",
    "\n",
    "z1=sigmoid.(W*x + b)\n",
    "z2=sigmoid.(U*z1 + c)\n",
    "N=exp.(V*z2) |> only\n",
    "render(N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$(\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(c_{1} + \\frac{U_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(c_{2} + \\frac{U_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(c_{3} + \\frac{U_{31}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{32}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{33}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))}) * (\\frac{(\\mathrm{exp}(-(c_{1} + \\frac{U_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})) * V_{11} * (\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * U_{11})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * U_{12})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * U_{13})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}}))}{(1 + \\mathrm{exp}(-(c_{1} + \\frac{U_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))^{2}} + \\frac{(\\mathrm{exp}(-(c_{2} + \\frac{U_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})) * V_{12} * (\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * U_{21})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * U_{22})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * U_{23})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}}))}{(1 + \\mathrm{exp}(-(c_{2} + \\frac{U_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))^{2}} + \\frac{(\\mathrm{exp}(-(c_{3} + \\frac{U_{31}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{32}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{33}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})) * V_{13} * (\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * U_{31})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * U_{32})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * U_{33})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}}))}{(1 + \\mathrm{exp}(-(c_{3} + \\frac{U_{31}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{32}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{33}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))^{2}}))$"
      ],
      "text/plain": [
       "L\"$(\\mathrm{exp}(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(c_{1} + \\frac{U_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(c_{2} + \\frac{U_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(c_{3} + \\frac{U_{31}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{32}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{33}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))}) * (\\frac{(\\mathrm{exp}(-(c_{1} + \\frac{U_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})) * V_{11} * (\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * U_{11})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * U_{12})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * U_{13})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}}))}{(1 + \\mathrm{exp}(-(c_{1} + \\frac{U_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))^{2}} + \\frac{(\\mathrm{exp}(-(c_{2} + \\frac{U_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})) * V_{12} * (\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * U_{21})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * U_{22})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * U_{23})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}}))}{(1 + \\mathrm{exp}(-(c_{2} + \\frac{U_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))^{2}} + \\frac{(\\mathrm{exp}(-(c_{3} + \\frac{U_{31}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{32}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{33}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})) * V_{13} * (\\frac{(\\mathrm{exp}(-(b_{1} + (x * W_{11}))) * W_{11} * U_{31})}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x * W_{21}))) * W_{21} * U_{32})}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x * W_{31}))) * W_{31} * U_{33})}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))^{2}}))}{(1 + \\mathrm{exp}(-(c_{3} + \\frac{U_{31}}{(1 + \\mathrm{exp}(-(b_{1} + (x * W_{11}))))} + \\frac{U_{32}}{(1 + \\mathrm{exp}(-(b_{2} + (x * W_{21}))))} + \\frac{U_{33}}{(1 + \\mathrm{exp}(-(b_{3} + (x * W_{31}))))})))^{2}}))$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dNdx = diff(N,x)\n",
    "render(dNdx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO workout the clean form of this"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find the parts that count to the gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mUndefVarError: @capture not defined\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mUndefVarError: @capture not defined\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:515\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "using MacroTools: postwalk\n",
    "tree = parse(repr(dNdx))\n",
    "\n",
    "tree = postwalk(x -> @capture(x, exp(z_)) ? 1 : x, tree)\n",
    "tree = postwalk(x -> @capture(x, (1+1)^2) ? 1 : x, tree)\n",
    "tree = postwalk(x -> @capture(x, z_/1) ? z : x, tree)\n",
    "tree = postwalk(x -> @capture(x, 1*z_) ? z : x, tree)\n",
    "tree = postwalk(x -> @capture(x, 1*z_*y_) ? :($z*$y) : x, tree)\n",
    "tree |> render"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "V_11*(W_11*U_11 + W_21*U_12 + W_31*U_13) + V_12*(W_11*U_21 + W_21*U_22 + W_31*U_23) + V_13*(W_11*U_31 + W_21*U_32 + W_31*U_33)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V*(U*W )|> only"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Note that each product term is scaled with a sigmoidal term that varies both with the value of the weights in that term and te corresponding biases, as well as with $x$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-output Multi-input network\n",
    "\n",
    "matrix equations actually come out all the same as before\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))))})$"
      ],
      "text/plain": [
       "L\"$(\\frac{V_{11}}{(1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))))} + \\frac{V_{12}}{(1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))))} + \\frac{V_{13}}{(1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))))})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/latex": [
       "$(\\frac{V_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))))} + \\frac{V_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))))} + \\frac{V_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))))})$"
      ],
      "text/plain": [
       "L\"$(\\frac{V_{21}}{(1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))))} + \\frac{V_{22}}{(1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))))} + \\frac{V_{23}}{(1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))))})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2-element Array{Void,1}:\n",
       " nothing\n",
       " nothing"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = vector_var(\"x\", 2)\n",
    "W = matrix_var(\"W\", 2, 3)\n",
    "b = vector_var(\"b\",3)\n",
    "V = matrix_var(\"V\", 3, 2)\n",
    "\n",
    "z=(sigmoid.(W*x + b))\n",
    "N=V*z\n",
    "render.(N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$(\\frac{(\\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))) * W_{11} * V_{11})}{(1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))) * W_{21} * V_{12})}{(1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))) * W_{31} * V_{13})}{(1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))))^{2}})$"
      ],
      "text/plain": [
       "L\"$(\\frac{(\\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))) * W_{11} * V_{11})}{(1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))) * W_{21} * V_{12})}{(1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))) * W_{31} * V_{13})}{(1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))))^{2}})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/latex": [
       "$(\\frac{(\\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))) * W_{12} * V_{21})}{(1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))) * W_{22} * V_{22})}{(1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))) * W_{32} * V_{23})}{(1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))))^{2}})$"
      ],
      "text/plain": [
       "L\"$(\\frac{(\\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))) * W_{12} * V_{21})}{(1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))) * W_{22} * V_{22})}{(1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))))^{2}} + \\frac{(\\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))) * W_{32} * V_{23})}{(1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))))^{2}})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2-element Array{Void,1}:\n",
       " nothing\n",
       " nothing"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dNdx = diff.(N,x)\n",
    "render.(dNdx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$(\\frac{(W_{11} * V_{11})}{((1 + \\mathrm{exp}(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))) * (1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12})))))} + \\frac{(W_{21} * V_{12})}{((1 + \\mathrm{exp}(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))) * (1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22})))))} + \\frac{(W_{31} * V_{13})}{((1 + \\mathrm{exp}(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))) * (1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32})))))})$"
      ],
      "text/plain": [
       "L\"$(\\frac{(W_{11} * V_{11})}{((1 + \\mathrm{exp}(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))) * (1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12})))))} + \\frac{(W_{21} * V_{12})}{((1 + \\mathrm{exp}(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))) * (1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22})))))} + \\frac{(W_{31} * V_{13})}{((1 + \\mathrm{exp}(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))) * (1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32})))))})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/latex": [
       "$(\\frac{(W_{12} * V_{21})}{((1 + \\mathrm{exp}(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))) * (1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12})))))} + \\frac{(W_{22} * V_{22})}{((1 + \\mathrm{exp}(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))) * (1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22})))))} + \\frac{(W_{32} * V_{23})}{((1 + \\mathrm{exp}(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))) * (1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32})))))})$"
      ],
      "text/plain": [
       "L\"$(\\frac{(W_{12} * V_{21})}{((1 + \\mathrm{exp}(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12}))) * (1 + \\mathrm{exp}(-(b_{1} + (x_{1} * W_{11}) + (x_{2} * W_{12})))))} + \\frac{(W_{22} * V_{22})}{((1 + \\mathrm{exp}(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22}))) * (1 + \\mathrm{exp}(-(b_{2} + (x_{1} * W_{21}) + (x_{2} * W_{22})))))} + \\frac{(W_{32} * V_{23})}{((1 + \\mathrm{exp}(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32}))) * (1 + \\mathrm{exp}(-(b_{3} + (x_{1} * W_{31}) + (x_{2} * W_{32})))))})$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2-element Array{Void,1}:\n",
       " nothing\n",
       " nothing"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdZm1dx = (W'.*V)*(sigmoid.(-(W*x + b)).*sigmoid.(W*x + b)) \n",
    "render.(mdZm1dx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Proof of Ability to Approximate any PDF\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\frac{1}{((1 + \\mathrm{exp}(z)) * (1 + \\mathrm{exp}(-(z))))}$"
      ],
      "text/plain": [
       "L\"$\\frac{1}{((1 + \\mathrm{exp}(z)) * (1 + \\mathrm{exp}(-(z))))}$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/latex": [
       "$(2 + \\mathrm{exp}(z) + \\mathrm{exp}(-(z)))^{-1}$"
      ],
      "text/plain": [
       "L\"$(2 + \\mathrm{exp}(z) + \\mathrm{exp}(-(z)))^{-1}$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diff(sigmoid(z), z) ≖ sigmoid(z) * sigmoid(-z) = true\n"
     ]
    }
   ],
   "source": [
    "@vars z\n",
    "@show diff(sigmoid(z),z) ≖ sigmoid(z)*sigmoid(-z)\n",
    "sigmoid(z)*sigmoid(-z) |> render\n",
    "(sigmoid(z)*sigmoid(-z)) |> ugly_simplify |> render"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\frac{(((-1 - (2 * \\mathrm{exp}(z))) - \\mathrm{exp}(2 * z)) + \\mathrm{exp}(z + z^{2}))}{((2 * \\mathrm{exp}(z + z^{2})) + \\mathrm{exp}((2 * z) + z^{2}) + \\mathrm{exp}(z^{2}))}$"
      ],
      "text/plain": [
       "L\"$\\frac{(((-1 - (2 * \\mathrm{exp}(z))) - \\mathrm{exp}(2 * z)) + \\mathrm{exp}(z + z^{2}))}{((2 * \\mathrm{exp}(z + z^{2})) + \\mathrm{exp}((2 * z) + z^{2}) + \\mathrm{exp}(z^{2}))}$\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "basic_simplify((sigmoid(z)*sigmoid(-z)) - exp(-z^2)) |> render"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our approximation to a PDF is:\n",
    "$$f(x)=\\frac{h(x)}{\\int_{S}h(z)dz}=\\frac{\\frac{\\partial N(x,\\tilde{p})}{\\partial x}}{N(max(S),\\tilde{p})-N(min(S),\\tilde{p})}$$.\n",
    "\n",
    "As we have shown above the numerator $\\frac{\\partial N(x,\\tilde{p})}{\\partial x} = g(x,\\tilde{p}) \\;N(x,\\tilde{p})$.\n",
    "\n",
    "Where $g(x,\\tilde{p})$ is  effectively a weighted sum of terms of forms similar to $\\frac{\\partial \\sigma(z)}{\\partial z} = \\sigma(z)\\sigma(-z) = \\frac{1}{(1+\\exp(-z)(1 + \\exp(-z))}$.\n",
    "As shown below that function defines a bell-curve.\n",
    "\n",
    "The weights and baises of the penultimate layers of the network ($W$, $U$), define the shape and position of that bell-curve.\n",
    "The  weight component of final expodential layer ($V$), determines the weighting of each bell-curve (though it also must scaled to counter effects of the prior layer weights).\n",
    "\n",
    "\n",
    "$$f(x)=\\frac{h(x)}{\\int_{S}h(z)dz} = \\frac{g(x,\\tilde{p}) \\; N(x,\\tilde{p})}{N(max(S),\\tilde{p})-N(min(S),\\tilde{p})}$$.\n",
    "For now we will focus on $g(x,\\tilde{p})$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 600 400\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip00\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"600\" height=\"400\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip00)\" points=\"\n",
       "0,400 600,400 600,0 0,0 \n",
       "  \" fill=\"#ffffff\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip01\">\n",
       "    <rect x=\"120\" y=\"0\" width=\"421\" height=\"400\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip00)\" points=\"\n",
       "38.8117,375.813 580.315,375.813 580.315,11.811 38.8117,11.811 \n",
       "  \" fill=\"#ffffff\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip02\">\n",
       "    <rect x=\"38\" y=\"11\" width=\"543\" height=\"365\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:0.5; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  173.052,370.353 173.052,17.2711 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:0.5; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  309.966,370.353 309.966,17.2711 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:0.5; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  446.881,370.353 446.881,17.2711 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:0.5; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  46.9343,311.362 572.192,311.362 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:0.5; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  46.9343,236.46 572.192,236.46 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:0.5; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  46.9343,161.558 572.192,161.558 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:0.5; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  46.9343,86.656 572.192,86.656 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  38.8117,375.813 580.315,375.813 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  173.052,375.813 173.052,370.353 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  309.966,375.813 309.966,370.353 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  446.881,375.813 446.881,370.353 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  38.8117,375.813 38.8117,11.811 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  38.8117,311.362 46.9343,311.362 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  38.8117,236.46 46.9343,236.46 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  38.8117,161.558 46.9343,161.558 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  38.8117,86.656 46.9343,86.656 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 173.052, 389.613)\" x=\"173.052\" y=\"389.613\">-2.5</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 309.966, 389.613)\" x=\"309.966\" y=\"389.613\">0.0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 446.881, 389.613)\" x=\"446.881\" y=\"389.613\">2.5</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 32.8117, 315.862)\" x=\"32.8117\" y=\"315.862\">0.05</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 32.8117, 240.96)\" x=\"32.8117\" y=\"240.96\">0.10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 32.8117, 166.058)\" x=\"32.8117\" y=\"166.058\">0.15</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 32.8117, 91.156)\" x=\"32.8117\" y=\"91.156\">0.20</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#009af9; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  38.8117,375.813 41.4859,375.298 66.2124,369.185 90.9389,359.788 105.798,351.921 120.658,341.869 133.222,331.292 145.785,318.457 159.433,301.562 173.08,281.199 \n",
       "  185.662,259.098 198.244,233.724 211.733,203.084 225.222,169.533 240.851,128.676 256.479,88.5685 263.424,71.9973 270.368,56.7333 277.312,43.1626 284.257,31.653 \n",
       "  290.346,23.5184 296.436,17.4114 302.525,13.4771 308.615,11.811 315.254,12.6255 321.893,16.1599 328.533,22.3119 335.172,30.9069 341.851,41.7788 348.529,54.5934 \n",
       "  355.208,69.0265 361.887,84.7361 376.655,122.309 391.424,161.043 405.475,196.513 419.527,229.043 426.832,244.489 434.138,258.848 441.443,272.098 448.748,284.248 \n",
       "  461.047,302.304 473.346,317.554 488.362,332.803 503.377,344.891 517.493,353.874 531.608,360.979 554.221,369.337 576.835,374.976 580.315,375.661 \n",
       "  \"/>\n",
       "<polygon clip-path=\"url(#clip00)\" points=\"\n",
       "489.799,62.931 562.315,62.931 562.315,32.691 489.799,32.691 \n",
       "  \" fill=\"#ffffff\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  489.799,62.931 562.315,62.931 562.315,32.691 489.799,32.691 489.799,62.931 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#009af9; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  495.799,47.811 531.799,47.811 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:start;\" transform=\"rotate(0, 537.799, 52.311)\" x=\"537.799\" y=\"52.311\">y1</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using Plots\n",
    "plot(x->sigmoid(x)*sigmoid(-x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### It has been shown in [Bacharoglou, 2010, APPROXIMATION OF PROBABILITY DISTRIBUTIONS\n",
    "BY CONVEX MIXTURES OF GAUSSIAN MEASURES](http://www.mathaware.org/journals/proc/2010-138-07/S0002-9939-10-10340-2/S0002-9939-10-10340-2.pdf)\n",
    "that every PDF with compact support can be approximated by an weighted non-negative sum of gaussians. \n",
    "(It actually does it with a restricted subset of gaussians, but we don't need that)\n",
    "\n",
    "\n",
    "Bacharogluo,2010 theorem 1 is noted as applicable to mixtures of any approximate identity ([defn](http://people.math.gatech.edu/~heil/7338/fall09/approxid.pdf)).   \n",
    "It is easy to construct an approximate identity from sequences of $\\frac{\\partial \\sigma(Wz+b)}{\\partial z}$,\n",
    "(though that depends on which form or approximate identity is there are many definitions)\n",
    "\n",
    "Which directly gives us the result that $g(x, \\tilde{p})$ can approximate any PDF with continuous support.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.0",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
